{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from attention_visualization import *\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from matplotlib.font_manager import FontProperties\n",
    "font = FontProperties(fname='./SimHei.ttf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./output_wiki_bert/bert.model.epoch.3 loaded for evaluation!\n"
     ]
    }
   ],
   "source": [
    "model = Pretrainer(vocab_size=config[\"vocab_size\"],\n",
    "                   max_seq_len=256,\n",
    "                   batch_size=1,\n",
    "                   )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[   2,  132, 2112, 1236,   40,  179,  754,    3]])\n"
     ]
    }
   ],
   "source": [
    "text = \"为什么要上班\"\n",
    "aa = model(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 12, 8, 8)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aa[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_attention(text, attention_matrices, layer_num, head_num):\n",
    "    labels = [i + \" \" for i in list(text)]\n",
    "    labels = [\"#CLS# \",] + labels + [\"#SEP# \",]\n",
    "    plt.figure(figsize=(8,8))\n",
    "    plt.imshow(aa[layer_num][0][head_num])\n",
    "    plt.yticks(range(len(labels)), labels, fontproperties=font, fontsize=18)\n",
    "    plt.xticks(range(len(labels)), labels, fontproperties=font, fontsize=18)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgEAAAHcCAYAAABPrnkNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xm0ZGV57/Hv0+f0ALSAyNDNICAqUVTUBY6goteoV0KcEzRXRBCN4kREjYrBIXqjGHGWNiIhCVGXLtdVo0LECQiiDaIgCAg0zdBMKtBAT3Q/9493l10Wp/uc7j5Vu+q8389atWrXHp/qPmfXr9733ftEZiJJkuozq+0CJElSOwwBkiRVyhAgSVKlDAGSJFXKECBJUqUMAZIkVcoQIElSpQwBkiRVyhAgSVKlxtsuYNjMmb1Nzpu3fdtlTLtYO3PvDJnRdgV9EjPzja2bM3O/e4zdu6btEvpjhv4sAjBD75p716pbbs/MnSZbzxDQY9687Tnwca9vu4xpN3736rZL6Jscn5kfKjlrZp54795rm7ZL6JvtLrql7RL6Y2ys7Qr6Jtbc13YJffG9q0+6birrzcyzpyRJmpQhQJKkShkCJEmqlCFAkqRKGQIkSaqUIUCSpEoZAiRJqpQhQJKkShkCJEmqlCFAkqRKGQIkSaqUIUCSpEoZAiRJqpQhQJKkShkCJEmqlCFAkqRKGQIkSaqUIUCSpEoZAiRJqpQhQJKkSk0pBETEwyNiRURE8/qbEfHWnnUeGhHfjoi7I+K2iPhkRMxrlu0VERkRx05ynIiI10fEbyJiZURcGRFHTrDen0fEFV2vfxURL5zKe5EkScVUWwIeBlydmdn1+qrOwojYFTgX2Bt4OXAc8ArgXzexnvcDJwGfAg4FfgacGhFPn6Ceq5pjB/DQ7nokSdLkxje2MCLmArOBRwJLImI+JTg8BFgWEXMyczVwArAj8NTMvLrZdlvg0xFxArB6ivW8CfhSZn6m2ccPgUOA1wM/joitgDHgEcDSpp6FwFbALRExnpn3Tf3tS5JUr42GAOAU4Iiu18u7phcD74uI9wEvBc7vBIDGT5vnA4HzJiskIsYoH+Y7duZl5tqmmX9dM+u7QHerwN92Td8KHAmcNtmxJEnS5N0B/wgcDJxDaaI/mPKt/A/N9KnAHsCDgF/3bHs58BeUboJJZeZa4BvAyyLiXyJin2b+zzJzcbPaG5vjXgW8q5n+AHBZM/2dqRxLkiRNEgIy86rMPBeYC5zXTN8OLMnMczNzKbCgWf13Pdvem5nfzszrNqGeo4HTgVcDV0bEVzphoNnnJU0N2wE/aKaXA1c09dy6CceSJKlqUx0YuDtwfdf00q5lc5rntVtaTGYuz8wjKGMQTgNeCFwYEQd11omIOcBOG6lnk0XEMRGxOCIWr15zz5bsSpKkkbHREBARs5rL/BZQBgKOAw8GboiI8YiYxfpxAvN7tt0tIs6NiMM2tajM/E1mHgU8hTIe4PRmn2PAnpTAcfsE9cSmHqs53qLMPCAzD5gze5vN2YUkSSNnspaAU4EVzXrXAGuAtwBvaKbfC/yW8kH9sJ5tdwWeyvqWgo2KiEMj4tKI2LczrxkL8Elg74jYGTgbuJIyoHFVU8MLgI8200fcb8eSJGlCk4WAEymD8W6gjPI/kPIhfHwzvSgz7wF+CBwSETt1bXsIkJRr/afiNmA/SnDotoASRO4AXgt8GPh5Vz3LKfckOBD41hSPJUlS9TZ6iWBmLomIdcBlnRH6EbEQODMzL+la9e2UqwC+HREnAvsA/wCckZlLI2KvZr39IuLQCQ51NuWD/RzgpOZ+AJdTAsFRwMnN/QiuaLoAfpGZiyNiB+ABwLcyc/kE+5UkSRsw2X0CAPalfPvvBID59NydLzMvioiDgY8AXwfuBj4HvLtnX69rHr32bgLHYcAHgbcBO1O6IN7a7Ku7np90Td9sAJAkadNNGgIy881d08vYQBdCZl4IPGsDy5YAkw7ay8w7gGObx4bW+cuu6fMpdwyUJEmbyL8iKElSpQwBkiRVyhAgSVKlDAGSJFXKECBJUqUMAZIkVcoQIElSpQwBkiRVyhAgSVKlDAGSJFXKECBJUqUMAZIkVcoQIElSpQwBkiRVyhAgSVKlDAGSJFXKECBJUqUMAZIkVcoQIElSpcbbLmDYxIrVzP71dW2XMe3W3nlX2yX0zZpnPrbtEvrinZ8/ve0S+uLkJx7cdgl9s/Z3v2+7hP6IaLuCvomxsbZLaJUtAZIkVcoQIElSpQwBkiRVyhAgSVKlDAGSJFXKECBJUqUMAZIkVcoQIElSpQwBkiRVyhAgSVKlDAGSJFXKECBJUqUMAZIkVcoQIElSpQwBkiRVyhAgSVKlDAGSJFXKECBJUqUMAZIkVcoQIElSpYY6BETE/hEx1kwfFhFLIuLYtuuSJGkmGNoQEBFPBC4GDm9m7QTsCVzXWlGSJM0gQxsCMvMC4OfAO5pZOzTPv22nIkmSZpahDQGNjwK/ioitgQc1865vsR5JkmaM8bYL6BURjwTWAmsoLQE/B3YG9gBWAjtGxI49m63IzFsGWqgkSSNu6EIAcBaw20aWXzvBvK8DL+lPOZIkzUzD2B2wLzAXGMvMyMwAxoBbgZM787qW3QIsaa9cSZJG09C1BGTmPRPMPpjSJfCd7pnN5YM7AUu35JgRcQxwDMC8WfO3ZFeSJI2MYWwJmMgbgBuBH0TE9l3zd6a8hxu3ZOeZuSgzD8jMA+bEvC3ZlSRJI2PoQ0BE7A+8CPgscBpwVkR06l7YPG9RCJAkqUZDHQIiYi7wBeA24FPAIuAA4DXNKgua55sGX50kSaNt6MYEdETEbOB04EDgsMxcDpwTEd9g/T0DFgAJLGunSkmSRtdQhoCIeCDlsr9DgBMy81tdi1+SmdlM7wrclplrBl2jJEmjbui6AyLiMMoNgg4BPpiZH+xe3gkATUvBoTgeQJKkzTJUISAiTgT+H+XugEdn5gkTrHNKRFwH/B54InD+QIuUJGmGGLbugA9R7hb46cz85QbW+QHwMuBq4DzgPQOqTZKkGWWoQkBmrmb9yP8NrfMV4CuDqUiSpJlrqLoDJEnS4BgCJEmqlCFAkqRKGQIkSaqUIUCSpEoZAiRJqpQhQJKkShkCJEmqlCFAkqRKGQIkSaqUIUCSpEoZAiRJqpQhQJKkShkCJEmqlCFAkqRKGQIkSaqUIUCSpEqNt13A0MkkV61qu4ppF7Nn7n/1vJ9d1XYJfXH6rU9tu4S+uPll+7ZdQt/s9Pmftl1CX8ScOW2X0D/rsu0KWmVLgCRJlTIESJJUKUOAJEmVMgRIklQpQ4AkSZUyBEiSVClDgCRJlTIESJJUKUOAJEmVMgRIklQpQ4AkSZUyBEiSVClDgCRJlTIESJJUKUOAJEmVMgRIklQpQ4AkSZUyBEiSVClDgCRJlRraEBARz42IJ0wwf9eIODYiHt5GXZIkzRRDGwKAjwJnRcTCnvl7AJ8C/mzwJUmSNHOMt13ARCLiqcCjgLdm5rKexfc2z3cMtipJkmaWYW0JeCewHPjiBMuyeV43uHIkSZp5hi4ENOMADgUuy8zlbdcjSdJMNVQhICIC+FjPvAdExNxJthuPiG37WpwkSTPMsI0JOBo4CLika945wP4lH/yJc3rm3Qls39fqJEmaQYYmBETEQ4CTga8AN1DCAMArKHWual4/DPhmM/+iZt44sNHWgkmOfQxwDMC82GZzdyNJ0kgZmhAAXAt8GXgXcHxnZmb+unuliOjUvDQzfzMdB87MRcAigO3GdsxJVpckaUYYmhCQmQkcBTBB078kSZpmQzUwUJIkDY4hQJKkShkCJEmqlCFAkqRKDWsIiOYhSZL6ZFhDwDgwtoFl85rnOQOqRZKkGWlYQ8BWwNYbWGYIkCRpGgzNfQK6ZeYxG1l2LnYVSJK0xYa1JUCSJPWZIUCSpEoZAiRJqpQhQJKkShkCJEmqlCFAkqRKGQIkSaqUIUCSpEoZAiRJqpQhQJKkShkCJEmqlCFAkqRKGQIkSaqUIUCSpEoZAiRJqpQhQJKkSo23XcCwWbXLVlx31GPbLmPa7fGh89suoW/WrV3bdgl9cdtB2XYJfbFw16Vtl9A39+XM/D/L1avbLqFvVp25Z9sl9Mezp7aaLQGSJFXKECBJUqUMAZIkVcoQIElSpQwBkiRVyhAgSVKlDAGSJFXKECBJUqUMAZIkVcoQIElSpQwBkiRVyhAgSVKlDAGSJFXKECBJUqUMAZIkVcoQIElSpQwBkiRVyhAgSVKlDAGSJFVqZEJAROzcdg2SJM0kIxECImIH4MqIOCUi5rRdjyRJM8FIhADgOGA74LzMXN12MZIkzQRDHwIiYlfgjcBXMvP0Zt54u1VJkjT6hj4EAJ8B1gJvAoiIAM6JiPdFxNxWK5MkaYQNdQiIiJcALwDelpm3NrP/GngS8DRgrK3aJEkadUMbAiJiT+AU4AeZeWozbz7wT8C1wIsy894WS5QkaaQNZd96RGwHfBNI4FVdi04EFgIHZeYfBl+ZJEkzx9CFgIhYSAkAfwY8JzOvb+YfBLwFeHdmXjDNxzwGOAZgfNsHTueuJUkaWsPYHfBm4HHA/8nMHwFExPbAvwFnAh+Z7gNm5qLMPCAzDxjbZpvp3r0kSUNp6EJAZr4TeGxmfhUgImYBXwbuA16RmRkRsyPi6Ih4epu1SpI0yoauOwAgMy/tenky5WqAp2TmHc28XYAPASsj4jFd8yVJ0hQNXUtAt4h4P/A64MWZeVlnfmbeALwa2IMSEiRJ0iYayhAQEXMj4tPA3wOHZ+bZzfyxiNguInYHrgZ+BBwREc9vr1pJkkbTUHYHAD8HHg0sBz4QEZ8Atgc2NGpvUUTsZ7eAJElTN6wh4CxKALgGuB64CbgVuK15vhO4u3m8A3gf5c6Cp7VQqyRJI2koQ0Bmvm2q60bEPwPfzcyL+liSJEkzzlCGgE2RmSsAA4AkSZtoKAcGSpKk/jMESJJUKUOAJEmVMgRIklQpQ4AkSZUyBEiSVClDgCRJlTIESJJUKUOAJEmVMgRIklQpQ4AkSZUyBEiSVClDgCRJlTIESJJUKUOAJEmVMgRIklSp8bYLGDaRMOu+tquQYNac2W2X0Bdrd9q+7RL6ZnyG/p/94QkL2i6hb2adkm2X0CpbAiRJqpQhQJKkShkCJEmqlCFAkqRKGQIkSaqUIUCSpEoZAiRJqpQhQJKkShkCJEmqlCFAkqRKGQIkSaqUIUCSpEoZAiRJqpQhQJKkShkCJEmqlCFAkqRKGQIkSaqUIUCSpEoZAiRJqpQhQJKkSo23XUC3iNgKWAeszsyc4jazgDlAZuaqftYnSdJMMmwtAf8JrATWRURO5QGsBVYAf99q5ZIkjZihagkAjgPeTflQX9fMexewL3BE8/p84GPA15rXY8BWwO8GV6YkSaNvqEJAZl7TOy8i7gJWZOaS5vVa4PbOa0mStHmGKgQARMSzgO9PML97jMCXIuJLXa+fnJk/7XtxkiTNIMM2JgBKVwDA/sDewBeAnzTTewM3A8c3009o1l054BolSRp5Q9cSwPqxAEsz846NdQdEROfDf0pXEkiSpPWGMQRs3Tz/ISL+OHOS7oB5W3LAiDgGOAZgfNsHbsmuJEkaGcMYAv6H0tTf7W+AS4GLN7DNsi05YGYuAhYBbLVwD1sVJElVGJoQEBFjwGxgVffI/4jYA3gPcCFwSGaunmDb+RGxM3BjZq7rXS5Jku5vaEIAcDDwQ4DuboAuTwFWbWBZxwOBO6a9MkmSZqBhujrgAuAhwEJgp+bxeeBO4Eld8x4N3EO5sVBn3u7Aw4C7Bl61JEkjamhaAjJzBXBt53VEnAC8Dvg65Y6AKym3CP405ZLBk6f69wUkSdL9DU0ImMDXgBuBJ1M++PcFfg9sDzzbACBJ0pYZ2hCQmZcDl0fEeZS/G/AGyhUC48API2IZ8B1KS8HZmXlfa8VKkjSChioERMTzgedRrvtfQOn/3x04Gzg8M7/TrLcL8ELgSOA1wO8i4pWd5ZIkaXJDFQKAxcDHgSXAFcBXgTMz85bulZrXnwc+HxGPA54LnDnYUiVJGm1DFQKaD/eHb+I2vwB+0Z+KJEmauYbpEkFJkjRAhgBJkiplCJAkqVKGAEmSKmUIkCSpUoYASZIqZQiQJKlShgBJkiplCJAkqVKGAEmSKmUIkCSpUoYASZIqZQiQJKlShgBJkiplCJAkqVKGAEmSKjXedgHDZvbN97DbRy9ouwxtgrzvvrZL6IuZ+r5mXbmk7RL65oaj92+7hL54wst/2XYJfXPj/57bdgmtsiVAkqRKGQIkSaqUIUCSpEoZAiRJqpQhQJKkShkCJEmqlCFAkqRKGQIkSaqUIUCSpEoZAiRJqpQhQJKkShkCJEmqlCFAkqRKGQIkSaqUIUCSpEoZAiRJqpQhQJKkShkCJEmqlCFAkqRKjUQIiIjXRsSlEbFL27VIkjRTjEQIAB4A7Afc13YhkiTNFKMSAlY2z6tarUKSpBlkVEKAJEmaZuNtF7CJtomI7tezgXnAXZl5TzslSZI0mkatJeBmYHnX4/fATcBL2yxKkqRRNGotAQuA7m/8s4A5PfMkSdIUjFoIuCcz757unUbEMcAxAPPYerp3L0nSUBq17oC+yMxFmXlAZh4wm7ltlyNJ0kAYAiRJqpQhQJKkShkCJEmqlCFAkqRKDdXVARFxOOUGQKt7Fj2+eX5xREx06+AxgMz8jz6WJ0nSjDJUIQD4MLAjE/+hoDuBT2xgu3HgbsAQIEnSFA1VCMjMvdquQZKkWjgmQJKkShkCJEmqlCFAkqRKGQIkSaqUIUCSpEoZAiRJqpQhQJKkShkCJEmqlCFAkqRKGQIkSaqUIUCSpEoZAiRJqpQhQJKkShkCJEmqlCFAkqRKGQIkSaqUIUCSpEoZAiRJqlRkZts1DJVtt9k1n7Tfa9suY9rl4kvbLqF/ItquoD9m6O9mzJ7Tdgl9E2N+rxo1373mp22X0BdjC397YWYeMNl6/sRKklQpQ4AkSZUyBEiSVClDgCRJlTIESJJUKUOAJEmVMgRIklQpQ4AkSZUyBEiSVClDgCRJlTIESJJUKUOAJEmVMgRIklQpQ4AkSZUyBEiSVClDgCRJlTIESJJUKUOAJEmVMgRIklQpQ4AkSZUaqRAQETtGxHjbdUiSNBMMVQiIiPkRMafr9VYRsXXXKhcBZ0bE1hExJyKiWW+82TYGXbMkSaNqqEIA8CNgVURkRCRwL/DJruX3AdcApwOrgHXNemuA5cAugy1XkqTRNWwh4IXAnsANwNuABwPv6lq+BrgLOBbYHVgGvAXYFXgIcNsgi5UkaZQNVQjIzOszcymwFrgzM68HxiPidV3dBPdm5s2ZeSOwrllvWWZem5lr26pdkqRRM1QhYAMeDHwOiOaxot1yJEmaGUYhBKxpnldTQsDKFmuRJGnGGKrL7SLiUcAcYDawe0Q8CRgDyMyMiG2BFRGxW7POGLBDROwDrG66DzbnuMcAxwDMm7Pdlr8RSZJGwLC1BHwc+CmwkDIg8Dxgb4CImA3sQLli4BPAtcAC4GPAbyldBpslMxdl5gGZecDs8a0n30CSpBlgqEJAZj47M+cAS4HXZ+ZYMw0lGIwDNwJHAvOb6aOBbYCXD75iSZJG11B1B2zA+cBOwAHN66szczlAc2+gtZl5b0u1SZI0soaqJWAimbkmM28HngYsy8xr265JkqSZYOhbAiJiHuUKgZcA/91yOZIkzRhDFwKaKwBmA6+KiDcB3wcuBB4GHNVmbZIkzSRDEwIiYk/g28AjKfcDWAZ8l/L3BE4BfgZcEBGPp9wrYB2l/oURsR+wFXBVZt45+OolSRo9QxMCMvO6iLgNeAdwRmbeBBARP6ZcGfACyt8IuJBymWDnJkLvAP4emAc8Dzh7wKVLkjSShiYEAGTmMyeYfQTwjMy8KCJmAVtlpncNlCRpCw1VCJhIZi4BTmum1+FtgyVJmhZDf4mgJEnqD0OAJEmVMgRIklQpQ4AkSZUyBEiSVClDgCRJlTIESJJUKUOAJEmVMgRIklQpQ4AkSZUyBEiSVClDgCRJlTIESJJUKUOAJEmVMgRIklQpQ4AkSZUab7uAYRP3rWXsljvaLmPa5cP3abuEvokVq9ouQZtiVrRdQd+s2XWHtkvoi7HlM/d37OGnP67tEvrk76a0li0BkiRVyhAgSVKlDAGSJFXKECBJUqUMAZIkVcoQIElSpQwBkiRVyhAgSVKlDAGSJFXKECBJUqUMAZIkVcoQIElSpQwBkiRVyhAgSVKlDAGSJFXKECBJUqUMAZIkVcoQIElSpQwBkiRVyhAgSVKlphQCIuLhEbEiIqJ5/c2IeGvPOn8VERc36y2JiHf2LM8NPH7ULN+rZ/6aiLgiIk6MiLk9+/rziLii6/WvIuKFm/lvIElSlcanuN7DgKszM7teL+osjIijmtcnABcAhwIfjohlmfmvXfs5A/hUz77v6nl9CnAasA3wRODdwG7Aa3rquao5dgAP7byWJElTs9EQ0HwDnw08ElgSEfMprQcPAZZFxJzMXA28GfheZn6o2fTsiHgK8HqgOwQsy8yfTlLTDV3rnN3U8N6IeDuwEhgDHgEsbepZCGwF3BIR45l539TeuiRJdZusJeAU4Iiu18u7phcD7wNOpHxrf1DPtkcD229hfQCXNs/7ACcBT+9a9rdd07cCR1JaESRJ0iQmGxPwj8DBwDmUZvyDKd/u/9BMn9qs9zXgiRHxjYjYHyAzL8nMc3r2FxEx3vOISWrYtXm+EXhjc9yrgHc10x8ALmumvzPJviRJUmOjISAzr8rMc4G5wHnN9O3Aksw8NzOXNqu+B/gY8Hzg4og4MyIeN8EujwPW9Dxe3FtTEw62jYhnAccD38/MZU2wOBfYDvhBM70cuKKp59bN+DeQJKlKU71EcHfg+q7ppd0LM3NNZr6N0mT/ceAg4PyIeEnPfv4dOLDncXbPOu+jhIM7ge8DFwOv7CyMiDnAThurZ1NFxDERsTgiFq9eu2JLdiVJ0siYbGDgLGAOsIAyEHAceDBwQzO9LjPXddbPzOuB4yLis8BZwBcj4nuZeXezyi2ZuXiSmj5H6WZYA1yTmX8chxARY8CewFrg9q56zmum13ZdwTBlmbmI5mqH7ebussnbS5I0iiZrCTgVWNGsdw3lg/ktwBua6fdGxOMj4tKIeGZno8z8LfB+YFvKlQWb4qbMXJyZv+wOAI2zgSsp4WVVU8MLgI8200cgSZKmZLIQcCJlMN4NrG++v5LST38g5dvzjcB+wDN6tl3QPC+bnlIBeC3wYeDnXfUsB17RTH9rGo8lSdKMttHugMxcEhHrgMs6zfgRsRA4MzMv6awXEWcAx0fEvZSbBT2GMnr/a00XQcfCiHjSBIe6aCrFZuYVTbP/LzJzcUTsADwA+NYErQaSJGkjpnLHwH0p3/47AWA+978731GU7oJXA++lDNo7CfhIz3ovbx699ph6yewL/KRr+mYDgCRJm27SEJCZb+6aXsYEXQiZuZJyy+ATNrKfye4HADDpOpn5l13T51PuGChJkjaRf0VQkqRKGQIkSaqUIUCSpEoZAiRJqpQhQJKkShkCJEmqlCFAkqRKGQIkSaqUIUCSpEoZAiRJqpQhQJKkShkCJEmqlCFAkqRKGQIkSaqUIUCSpEoZAiRJqpQhQJKkShkCJEmqlCFAkqRKGQIkSapUZGbbNQyViLgNuG5Ah9sRuH1Axxq0mfrefF+jZ6a+N9/X6Bnke9szM3eabCVDQIsiYnFmHtB2Hf0wU9+b72v0zNT35vsaPcP43uwOkCSpUoYASZIqZQho16K2C+ijmfrefF+jZ6a+N9/X6Bm69+aYAEmSKmVLgCQNsYh4bURcGhG7tF2LZh5DgARExHMj4gkTzN81Io6NiIe3UVc/RMTObdewuSJiq4iYGxGxCdvMioh5ETG3n7X10QOA/YD72i6kHyJix4gYb7uOWhkCNiAiHh4RKzonm4j4ZkS8tWedh0bEtyPi7oi4LSI+GRHzmmV7RURGxLGTHCci4vUR8ZuIWBkRV0bEkROs9+cRcUXX619FxAun591umojYPyLGmunDImLJZO9zBHwUOCsiFvbM3wP4FPBngy9p+kXEDsCVEXFKRMxpu57N8J/ASmBd8/s16QNYC6wA/r7VyjffyuZ5VatVbKaImN/9s9YEua27VrkIODMito6IOV3n3PFm2ykHvi2scyrn/L+KiIub9ZZExDt7lm/o5/BHzfK9euaviYgrIuLE3pA6qHO+IWDDHgZcnesHTTwMuKqzMCJ2Bc4F9gZeDhwHvAL41008zvuBkygfNIcCPwNOjYinT1DPVc2xA3hodz2DEhFPBC4GDm9m7QTsyeBusDTtIuKpwKOAEzNzWc/ie5vnOwZbVd8cB2wHnJeZq9suZjMcR/m/2ofyu7c38AXgJ12vbwaO73r9UODRwCkt1Cv4EbCqK5TdC3yya/l9wDXA6ZSgs65Zbw2wHBhUN8hk5/yjgDOAr1LO1d8APhwRR/Ts5wzgyT2P1/esc0oz/7mUz4zjgU9PUE/fz/k2wfRo0ths4JHAkoiYTwlLDwGWRcSc5uR5AuXuT0/NzKubbbcFPh0RJwBTPcG+CfhSZn6m2ccPgUMoPzQ/joitgDHgEcDSpp6FwFbALRExnpkDaybMzAsi4ufAO4B/B3ZoFv12UDX0wTspJ5svTrCsc0JYN7hy+qMJrm8EvpKZpzfzBvrzs6Uy85reeRFxF7AiM5c0r9cCt3dezyDb9Hwpng3MA+7KzHvaKWlKXggEcB5wMuVDtLtVYw1wF+Wc+mbg58A/NevNA27rZ3GbcM5/M/C9zPxQs+nZEfEUyrm6+8vfssz86SSHvaFrnbObGt4bEW+ntPwM7JxvS8D9nUL5QPgI8Pxm+k5gDrAYeFeTyl4KnN8JAI3Of+qBUzlQ06S+FSVMAJCZaym/NB9tZn23qeENwN8201c2y24F/mbT3t60+Cjwq6ZJ70HNvOtbqGOLRRkHcChwWWYub7uePvsMpWn8TfDHbxfnRMT7epsih1lEPKunuf/vgOd0vd4N+FJPs+uT2q16WtyUyxvTAAAJqUlEQVRM+f3vPH4P3EQ5Fw2tzLw+M5dSfvbuzMzrgfGIeF1XN8G9mXlzZt5ICdx3ZuayzLy2OSf206Tn/Ga9bVh/vus4GnjbNNRwafO8DwM+5xsC7u8fgYOBcyhN9AdTkt4fmulTKf3EDwJ+3bPt5cBfULoJJtX8cH8DeFlE/EtE7NPM/1lmLm5We2Nz3KsoP4wHAx8ALmumv7NZ73ITRcQjI2LfiHgIJam/G9iZ8m+xEtix6e/qfgz1aObmQ/BjPfMeMNkHYtNXuW1fi5tmEfES4AXA2zLz1mb2XwNPAp5G+eYxKlY0z/szeXdAZ7DnSkbfAsogwc5jO0p33FfaLGozPRj4HKWFIFj/f9qGqZzzAb4GPDEivhER+wNk5iWZeU7P/qI5R3Q/JhvXsGvzfCMDPufbHdAjM68Crmo+CM7LzHOjDBZbkpnnwh+/PQL8rmfbe4FvN+vsNcVDHk05Qb0aODIivga8q9PCkJmXNPvbDvhB0xz/ZOCKTj0DchblG9aGXDvBvK8DL+lPOdPiaOAg4JKueecA+0/wO3tOz7w7ge37Wt00iYg9Kd92fpCZpzbz5lOaXK8FXtT87I6KTtfM0sy8Y2PdARHR+fCfCTdEuScz7267iGmypnleTQkBrYW0qZzzG++hhOU3AS+IiLOAd2bmL3p2eVzz6PZSSojomBXlioitKS3HxwPfb8YkLYPBnfNtCdiw3VnfxL07sLRrWacJa4ubqTJzeWYeQemPOo3SFXBhRBzUWadpMttpI/UMwr7AXGAsMyMzg/ILcStwcmde17JbgCUDrnHKmhaNkynfos7qWvQK4LGU/rhHAId1ze/MezTwrIEVuwWaE8k3KR+Cr+padCKln/HwzPzD4CvbIp2R5X+YrDuA5oRK6VtWCyLiURHxeEq/++5N18w8gGYQ3rbAiojYrfnyNAbsEBH7RMQeAyx1Y+d8MnNNZr6N0mT/ccoXiPObVrZu/075YO9+nN2zzvsoQehO4PuUwdav7Cwc5DnfloAeETGL8iG/gDIoZJzSdHVDM72O0kcDML9n290oHyofAX61KcfNzN8AR0XE5ygfSqcDD2nGDexJCRy3d9VzXjO9tms0a99sYODRwZQugT9pnmpq3onBB5VNcS3wZUpz2/GdmZn5J108sf765aXN/9HIaL7NfJNyeeNzmr5YmoD5FuDdmXlBiyVurv+hNPV3+xtKv+rFG9im96oPDc7HgadTPm/eRRkAeARARMymDC6+F/gE8OJmm481j/+ijNnpm6mc8zPzjwODm9+j4yLis5Rz9Rcj4ntdrTS3dHXnbsjnKN0Ma4BruscjDfycn5k+uh6Ub+O5kceJlAEia4Fv9Wx7YLPOS4C9muljN3KsQyknrn175p/YbLsz5fKajdXzqhb/rb4K3EBJ7tt3zV/Y1Pbitv8/p/g+TgJ+uoFlj2rey0Ft17kZ7+v/Ui6/elnXvO0pAei/aG4bPiqP5udsXm/drB+Xch4wZwPbzm/Wm9X2+9iM931s8zM4v+1atvB9LAGObqaf1rynBzfP/4syzmGb5pxyFKXFZ9sB1DWVc/7jm3P1M3u2fVWzzhOa1wmctJFjdT4X3rORdQZ6zrc74P5OpAzMuIH1TTlXUr4tHggsyvKt+IfAIRGxU9e2h1D+k342xWPdRrkT2FN75i+gDJS5A3gt8GHKYLxOPcspzdMHAt/apHc3TZqBMS8CPkv5JTqrSdRQQgCUQS5qSWa+E3hsZn4V/viN58uUYPCKzMyImB0RR8f970sxjA6m/F78yY2CKC1Oc4Gn0HU9es86y5v1RmpA5wx2PqW18JHN66uzdI12WhzXZua9mXnXAGo5kUnO+ZRz2X7AM3q2XdA8T2dL00DP+XYH9MgymGgd5ZKxxfDHZtUzsxmk13g75SqAb0fEiZR+on8AzsjMpV0DA/eLiImas86m/CefA5wU5X4Al1MCwVGUfvbVwBVNE9AvMnNxlDu+PYDSCtHKJW3NAJovUELMpyh96D8GXkMZgNb5xbipjfq0XmZe2vXyZMrVAE/JzM7Nj3YBPgSsjIjHdM0fRhdQrt1ewfpb6H6AcuOq5wCdy3UXUC7XPQH4t2beXMrluIP4UNEkMnMNpan7aZTr6icaWDyoWqZ0zo+IM4DjI+Jeys/iYyjdG1/LpqutsXADl6ReNMV6BnrONwRMbF+a6zKbH4b59NypKTMvioiDKf3/XwfupvTzvLtnX69rHr32bn74DgM+SLnWdGfKnbPe2uyru56fdE3f3GIAmE0Zr3AgcFhTxzkR8Q3WX0O7gNIiYj/skIiI91N+Dp+XmZd15mfmDRHxasq3i5P508GDQyUzV9B1FUqUm3K9jvL7txWlS2At5c5rP6EE6ZG5KiAiDqcMnuu90djjm+cXR8REtw4eA8jM/+hjedMqyu3V11C6Tv+75XJgCud8ypezayhXcr2XMmjvJMpnQLeXN49emzLIcWDnfP+UsKYsIh5IOeEeApyQmR/sWhadE25EvAd4Y2YO9X0COiLiJEqf//3Se0Q8inIJ4cE52Esyp0XTavMxShPjX2fm15v5Y5QTXeea889SmjoPzcz/aqfaTRMRj+BPb826L+UGOtsDz87Mn2xk86ETEUsoNw7b1LvBjQN3Z+aCSddsSXNfjcspIW5byoj4Cykj6Z+WXdfaR8QNlD7z01ootTq2BGhKmhaLf6Z0e3ywOwDAHy/16bQUHMpojQfo3LBkJvo55ZLG5cAHIuITlA/JbTaw/qKI2G/IuwUAyMzLgcsj4jzKaPM3UAZvjQM/jIhllCtXvg6cnUN+e+TM3KvtGqZbc4+Kb1P6/oPSOvhdyuC3Uyjjpy5oLiFcSbn6apzSpL4fpYXnqsy8c/DV18EQoEk1Yx7+gdJMeXRm3u8e+xFxCuWPYexA+Yb52UHWuIXG2fAd8zrXl4/iX9yDcgnTckoz5vWUcRq3UsZz3Eq5Tvnu5vEOyvXLL6AM9hw6EfF84HmU/5cFlICzO2WMzeGZ+Z1mvV0o99w4kjJW5XcR8crOcg1GZl4XEbdRfrbOyMybACLix5QBxC+g3C3vQsplgp2bCL2D8lcf51H+v3uvs9c0sTtAk2puXPEZ4NOZ+csNrPNXwOcpf03wPEpz3kjchCYiFlG6Ax45wbKDKIM3n5eZ3xt4cQPUDE59RGZOaQBTG5oP93Mol5tdQWnpODMzb9nINo+jBNSPZP/vQ68paAZOPyMzT+tcp5+ZM+HWziPHECBJUqW8T4AkSZUyBEiSVClDgCRJlTIESJJUKUOAJEmVMgRIklSp/w9xwOs/DDENsQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_attention(text,aa,  layer_num=2, head_num=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_attention(text, attention_matrices, head_num):\n",
    "    labels = [i + \" \" for i in list(text)]\n",
    "    labels = [\"#CLS# \",] + labels + [\"#SEP# \",]\n",
    "    plt.figure(figsize=(8,8))\n",
    "    attention_matrix = np.sum(np.concatenate(attention_matrices, axis=0), axis=0)/6.\n",
    "    print(attention_matrix.shape)\n",
    "    plt.imshow(attention_matrix[head_num])\n",
    "    plt.yticks(range(len(labels)), labels, fontproperties=font, fontsize=15)\n",
    "    plt.xticks(range(len(labels)), labels, fontproperties=font, fontsize=15)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(12, 8, 8)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfgAAAHYCAYAAAChhPpJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xm0JnV95/H3p3egsRsUHBZZXBJBo8T0oIgMJCox7lFiXKNxCDhqEo9xjqOjQkyM5LiMjmtax2icJAczSRQcUDGDy0jQaQiOaFABgcFGNulGaXr/zh9VLU9uerndfZ9b9/n1+3VOna6nqp6qb3Hvrc/zq9+vHlJVSJKktswbugBJkjTzDHhJkhpkwEuS1CADXpKkBhnwkiQ1yICXJKlBBrwkSQ0y4CVJapABL0lSgxYMXcBsmr//AbVw+cFDlzHjFv603W8jnLduw9AljEUtXjR0CdpN2bR56BLGYsMh7f4u1uKtQ5cwFht/sPqOqjpkV9vtUwG/cPnBHH3Wa4cuY8YddlmbIQiwZNW1Q5cwFlsfdtTQJWg3zb/lx0OXMBbXnt3u7+LWh9w7dAljcf0L3nTjdLbzFr0kSQ0y4CVJapABL0lSgwx4SZIaZMBLktQgA16SpAYZ8JIkNciAlySpQQa8JEkNMuAlSWqQAS9JUoMMeEmSGmTAS5LUIANekqQGGfCSJDXIgJckqUEGvCRJDTLgJUlqkAEvSVKDDHhJkhpkwEuS1KBpBXySlUme189/IcmJI+sek+TrSdYluTzJL/XLT0tSSZbuYJ9J8uYkP0zy4ySfSnLwlG1+dqwk30ty6J6eqCRJ+5LptuAfAVzdzx8PfBsgySHAJcDl/TYXABclOXAa+3wl8NvA04GTgAcD75myzfHAt5MsAQ6sqtumWa8kSfu0nQZ8ko8kWU8XwFf280cAdyY5GjgL+DHwmqr6AXAeEOBp0zj26cDnquqfquq7wFuAA/rjfmH0WMAa4IH9MkmStAu7asG/BXgscAdwAvAc4Kp+fjVwKvDVqiqAqtoKvJculHflGuA5SU7u33tRVT23X3fmlGP9LnBRPy9JknZhpwFfVbcA64Abq+oaYBNwbVVdU1WbgCOBW6a8521Vdck0jv1Wutv7X03ytSRPGdnHTaPH6hf/88j8tCU5K8mqJKu2rLtnd98uSdJEmk4f/FHAjduZB1gEbNmTA1fVPVX1EuA44Lt0fffnTvO4u3OclVW1oqpWzN//gD3ZhSRJE2fBzlYm+SDwUmBBkjXA4n75mcCjgLuAqSPfvwH8LfD1Xex7BfD9vv/95Um+AnwsyX8DPgQ8ceRY+wGbk/xxVS3f7bOUJGkfM50++I8Cb6fr/76Irj98Wx/8KroBeAAkmQ88DLhpGsf+B+AZI68vohugdxBdH/zosa4Cnot98JIkTcuu+uDvAA4EvllVN9CF7xVVdUNVbQY+ADwyyTn9qPrz+rdePLKbo5IcMzId1i+/EHh9/xz9tvdeT9fX/qPRYwGHAF/v5yVJ0i5Mpw/+aO5rkY/OU1VXA08Bnk03Kv5k4ElVtWbk/d8GfjAy/X2//NV0t/EvBr4JHA48vR+897NjJZkHLKuqu3b77CRJ2kfttA8eoKqeODL/kO2svxT4xe0s/xLdLfcd7XcN3a34Ha0fPdb9d1WnJEm6j99FL0lSgwx4SZIaZMBLktQgA16SpAYZ8JIkNciAlySpQQa8JEkNMuAlSWqQAS9JUoMMeEmSGmTAS5LUIANekqQGGfCSJDXIgJckqUEGvCRJDTLgJUlqkAEvSVKDDHhJkhpkwEuS1CADXpKkBi0YuoDZtOiWezjqrZcNXcaMm//AQ4cuYWzuOeXnhy5hLH7lj//30CWMxVdee9LQJYxNNhw4dAljUQ2nwJGfaPPkrp/mdrbgJUlqkAEvSVKDDHhJkhpkwEuS1CADXpKkBhnwkiQ1yICXJKlBBrwkSQ0y4CVJapABL0lSgwx4SZIaZMBLktQgA16SpAYZ8JIkNciAlySpQQa8JEkNMuAlSWqQAS9JUoMMeEmSGmTAS5LUIANekqQGzYmATzIvybOS7Ne/vjTJ+UPXJUnSpJoTAQ8sAj4EnN2/Phy4YbBqJEmacHMi4KtqPfAe4EX9ooOBHwxXkSRJk23QgE+vf/lB4NT+9UHALcNVJknSZFsw8PFPBr6a5F5gY78swHzgk0m2jixbDDywqtbOfpmSJE2WoW/Rfw2YX1X7V9XyqloOvBS4E1g+suwMYP2ehHuSs5KsSrJqExtmtnpJkuaoQQO+OlunLH4h8Jkpyw8DbtrDY6ysqhVVtWIhi/e0VEmSJsrQLfh/IcnDgF8HLkny3iTb6jsM+OFwlUmSNFnmTMAnmQ+sBD4LXEwX9K/qVxvwkiTthqEH2QGQZCFduB8H/GJVrU3yBuCYfpPDgH8eqDxJkibO4AGf5FDgfOBRwJOr6haAqvrLkc2OAb44+9VJkjSZBg34JK8H3gjcCpxSVd8ZWXci8FjgKODfAr8/SJGSJE2goVvw36Jrvb+uqu6esu5u4O10X1l7blVdPsu1SZI0sQYN+Kq6CLhoB+uuAZbObkWSJLVhzoyilyRJM8eAlySpQQa8JEkNMuAlSWqQAS9JUoMMeEmSGmTAS5LUIANekqQGGfCSJDXIgJckqUEGvCRJDTLgJUlqkAEvSVKDDHhJkhpkwEuS1CADXpKkBhnwkiQ1yICXJKlBBrwkSQ1aMHQBsynz5jFv/wOGLmPGbX3QoUOXMDYHfOOGoUsYi6/c/tChSxiLLYvbbTNkSw1dwlgseNhPhi5hbHLpkqFLGFS7f42SJO3DDHhJkhpkwEuS1CADXpKkBhnwkiQ1yICXJKlBBrwkSQ0y4CVJapABL0lSgwx4SZIaZMBLktQgA16SpAYZ8JIkNciAlySpQQa8JEkNMuAlSWqQAS9JUoMMeEmSGmTAS5LUoDkR8EnmRB2SJLVirgTrmUm+meTQ0YVJNiV51lBFSZI0qeZKwL8Q+H5V3TZl+b3A2gHqkSRpoi0YuoAkjwZO7aeptgKbZ7ciSZIm31xowb8RoKq+MnQhkiS1YtAWfJITgN8Yef0Aulb71m2LgKVJlvev9wNurypb9ZIk7cRgLfh+5PwHgK+NLP4ScCdwVz/dD7h45PVq4KGzWqgkSRNoyFv0jwOOBV49suxEYGFVpapCN8DulJHXS4Dv7s5BkpyVZFWSVRtr/UzVLknSnDbYLfqquqy/Rb9oZNm6Xbxnwx4cZyWwEmDZ/AfU7r5fkqRJNOggu+08FidJkmbAXBhFL0mSZpgBL0lSg+ZCwM8DSJIdrJsLNUqSNFHmQngu7v9dsp11+/WTJEnaDYN/VW1VfZ/uC222t27hLJcjSVIT5kILXpIkzTADXpKkBhnwkiQ1yICXJKlBBrwkSQ0y4CVJapABL0lSgwx4SZIaZMBLktQgA16SpAYZ8JIkNciAlySpQQa8JEkNMuAlSWqQAS9JUoMMeEmSGmTAS5LUIANekqQGGfCSJDVowdAFzKZaspg6/sFDlzHj5n33xqFLGJ95GbqCsdj8nmOGLmE8Gm4ybDp4/6FLGIvFC+8duoSxueHFW4cuYTw+P73NGv5zlCRp32XAS5LUIANekqQGGfCSJDXIgJckqUEGvCRJDTLgJUlqkAEvSVKDDHhJkhpkwEuS1CADXpKkBhnwkiQ1yICXJKlBBrwkSQ0y4CVJapABL0lSgwx4SZIaZMBLktQgA16SpAYZ8JIkNWjOBXySX07ysSRHDl2LJEmTasHQBWzHOcD9gLuHLkSSpEk1pwI+yVOBxwGPqaq7kxwBrK6qGrg0SZImypy5RZ/kQODDwJ9U1XeSzAM+B1yYJMNWJ0nSZJkzAQ+8G/gJcF7/+uXAccB7bcFLkrR75sQt+iRnAS8GTq6qjUkeAPwp8IdVdcmw1UmSNHkGD/gkvwN8ADizqq7sF38IuAp42wzs/yzgLIAli5bt7e4kSZoIgwZ8kmXAbwFvqKpP9MvOBE6mG2i3dW+PUVUrgZUA9zvgCG/1S5L2CYP2wVfVWuCXq+qdAElOouuLP6OqfpTk0CSXJjl1yDolSZo0gw+yq6rNAEmOAy4AXlVVl/XrbgPuAD6RZOlwVUqSNFkGD3iAJCuAS4F3VNUnp6x+NbAUeOesFyZJ0oQaPOCTPBu4DDgA+LUkVyS5NsltSe4FbgH2B85O8uQha5UkaVIMPooeuBI4H/gucDOwGri9n34M3Ev3QeQbwBMAH5uTJGkXBg/4qroJeMkuNtuS5PFVtWE2apIkadINfot+ugx3SZKmb2ICXpIkTZ8BL0lSgwx4SZIaZMBLktQgA16SpAYZ8JIkNciAlySpQQa8JEkNMuAlSWqQAS9JUoMMeEmSGmTAS5LUIANekqQGGfCSJDXIgJckqUEGvCRJDTLgJUlqkAEvSVKDFgxdwGyq+WHT/RYNXcaMW3TU4UOXMDbz7rp76BLG4oDv3Tl0CWORzVuGLmFsbn3iYUOXMBbHHnTz0CWMzbevPmjoEgZlC16SpAYZ8JIkNciAlySpQQa8JEkNMuAlSWqQAS9JUoMMeEmSGmTAS5LUIANekqQGGfCSJDXIgJckqUEGvCRJDTLgJUlqkAEvSVKDDHhJkhpkwEuS1CADXpKkBhnwkiQ1yICXJKlBBrwkSQ0y4CVJapABL0lSgwYN+CT7J1k4zW0XJtlv3DVJktSCBQMf/yrg8CSb+9f7AVuBDcCSftn6/t8FwD8Bp8xqhZIkTaBBA76qfm70dZIPA9dW1TuTvBP4aVWdO0hxkiRNsKFb8CT5NHA6sJG+BZ/kTfQt+CSvARYBX66qXxusUEmSJshcGGR3L/DaqloO/Dnw5n7+/cB5/fzvcd+t+t2S5Kwkq5Ks2rTpnhkrWpKkuWwuBPx0a8ie7LyqVlbViqpasXDhAXuyC0mSJs7gt+iBzcB5Sc7btqC/Rb9t/jX97BdnuzBJkibVYAGfZAEwr6peNLLsQcDf0d2y/+qU7ecluR9wb1Vtmt1qJUmaLEPeon8OsDbJ2iRrkqwFbgKOAy7sl21MsiHJGuAu4DbgpAFrliRpIgwW8FX1qarar6qWAQcB5wOXAvfvB9YdC9wOnF5Vy6tqWVUtqaqvDFWzJEmTYi70wQOcBjwO2B/4aJLLgRcDH6iqLw9ZmCRJk2gujKKnqi6tqkcBJ9I9Dvc+uhb8byb5kySPT7JHo+glSdoXDdqCT/I8uiA/EHg43dfQrgKeAPwj8FjgDODvgc1J3lVV7x6oXEmSJsbQt+jXAycD1wKfBl5VVbeOrL8cuDzJG+mCftXslyhJ0uQZ+rvoLwAumMZ2G4G/Gn9FkiS1YU70wUuSpJllwEuS1CADXpKkBhnwkiQ1yICXJKlBBrwkSQ0y4CVJapABL0lSgwx4SZIaZMBLktQgA16SpAYZ8JIkNciAlySpQQa8JEkNMuAlSWqQAS9JUoMMeEmSGrRg6AJmU36yjgWXXjV0GTNuy2MfOXQJY1Pfv3PoEsYi9z926BLG494auoKxWXbdxqFLGIu/ecjnhy5hbH7uiocMXcKgbMFLktQgA16SpAYZ8JIkNciAlySpQQa8JEkNMuAlSWqQAS9JUoMMeEmSGmTAS5LUIANekqQGGfCSJDXIgJckqUEGvCRJDTLgJUlqkAEvSVKDDHhJkhpkwEuS1CADXpKkBhnwkiQ1yICXJKlBBrwkSQ2acwGf5ItJ3jd0HZIkTbI5F/DAvcCGoYuQJGmSzcWA3zJ1QZIFSRYPUYwkSZNoLgY8wB8kqW0TsAn4+tBFSZI0KRYMXcAOvB9488jrRUAGqkWSpIkzVwN+Q1WtmYkdJTkLOAtgCfvPxC4lSZrz5uot+hlTVSurakVVrViI3fiSpH1D8wEvSdK+yICXJKlBBrwkSQ0abJBdkvnAQ4H1/Mtn35cAS5McOfUtwGLghqraPDtVSpI0mYYcRX8Q8C26b66rKeseBzx/yrLQhf+DgR+OvTpJkibYYAFfVXfQPd8uSZJmmH3wkiQ1yICXJKlBBrwkSQ0y4CVJapABL0lSgwx4SZIaZMBLktQgA16SpAYZ8JIkNciAlySpQQa8JEkNMuAlSWqQAS9JUoMMeEmSGmTAS5LUIANekqQGGfCSJDXIgJckqUEGvCRJDVowdAGz6oD94FGPGLqKGbfw/90xdAnjc9DyoSsYi1q3YegSxiLrNw5dwtjM27R16BLG4vg/f9XQJYzNdS//0NAljMX810xvO1vwkiQ1yICXJKlBBrwkSQ0y4CVJapABL0lSgwx4SZIaZMBLktQgA16SpAYZ8JIkNciAlySpQQa8JEkNMuAlSWqQAS9JUoMMeEmSGmTAS5LUIANekqQGGfCSJDXIgJckqUEGvCRJDTLgJUlqkAEvSVKD5mTAJ9kvyf5D1yFJ0qRaMNSBkwR4BnAvUMAi4PqqugZ4AfBfkpwCHA5sBtJvc3lV3TlM1ZIkTYbBAp4usH8dOBo4BvgC8DngGmATcDPwaOA04OXAB+gC/vuAAS9J0k4Mdou+qrZW1W8DfwH836p6BbA6ydnAFuCeqvok8Pv99q+uqrOq6ntD1SxJ0qSYa33wDwaeQte6XzdwLZIkTay5FvBb+mkeXd/8XktyVpJVSVZt2nTPTOxSkqQ5b8g+eJJcBywH9kuyGngvXbgvB9YlOQf4zX7bq4GNVfWY3TlGVa0EVgLcb+kRNYPlS5I0Zw3dgj8FOAe4FPgl4HZgCXAI8GPgI3Qj6gGeCbxwgBolSZo4gwZ8Va0GfgpsqqpbqupjVfVU4BfoHplbDVzXb7vtETpJkrQLQ7fg/5Uk8+kejbts4FIkSZpYg/bBj1ie5GXALXS35zdjwEuStMeG/Ca744HX0bXWjwTm0w2yOxd4H7AgydYp71kMbK2qTbNarCRJE2bIW/R3AsfRDbK7f1WdArysX/du4GLgJ3TfaLc2yRpgLd232kmSpJ0YrAVfVbcCJ01Z/AfA0qr6aZLTq2rjAKVJkjTx5kofPABV9d2RecNdkqQ9NOdG0UuSpL1nwEuS1CADXpKkBhnwkiQ1yICXJKlBBrwkSQ0y4CVJapABL0lSgwx4SZIaZMBLktQgA16SpAYZ8JIkNciAlySpQQa8JEkNMuAlSWqQAS9JUoMMeEmSGmTAS5LUoAVDFzCbsmEj86/94dBlzLi1pz106BLGZtkVPxq6hLHYdMTyoUsYi4XfuXnoEsZm4a13D13CWCx+5PqhSxibM6570tAljMm109rKFrwkSQ0y4CVJapABL0lSgwx4SZIaZMBLktQgA16SpAYZ8JIkNciAlySpQQa8JEkNMuAlSWqQAS9JUoMMeEmSGmTAS5LUIANekqQGGfCSJDXIgJckqUEGvCRJDTLgJUlqkAEvSVKDDHhJkhpkwEuS1KBpBXySlUme189/IcmJI+vOTnJ9kruTfD7J0f3yY5LUdqZz+/UfH1m2PsmVSU6f7nElSdKOTbcF/wjg6n7+eODbAEmeBrwdeCnwKGAL8Mkp7308cNDIdN7Iugv6ZQ8F/hr4TJJjd3VcSZK0czsN+CQfSbIeOAm4sp8/Arizb6mfDlxeVV+tqhuANwGbkywa2c1PqmrNyLR+ZN2mftnNVfUOYDXwq9M4riRJ2oldteDfAjwWuAM4AXgOcFU/vxq4Bjg1yTMAqurKqvqVqtq4h/VsBBZO47iSJGkndhrwVXULsA64saquATYB11bVNVW1CVgJ/Bnwd0m+leQF29nNZUnWjEwPm7pBknlJzgAeAlw6jeNOW5KzkqxKsmrj1vW7foMkSQ2YTh/8UcCN25mnqrZU1WuBY4FLgE8k+fiU959B1/LeNt0wsu6ZSdYA64EPA2dX1bY+9x0ed3dU1cqqWlFVKxbNW7Inu5AkaeIs2NnKJB+kG0C3oA/ixf3yM+kG1S0DflRVNwOvTXIBcGmSjwI397tZ3ffPb88XgVcC66vqR9M9blXdtCcnK0nSvmKnAU/XF74JuAv4OPAu4GK6YF4NXAj8DfDH/fZfpru1fjD3BfzOrNtB+O/quJIkaSd2GvBVdUeSA4EvVdUNSQ4CrtgWykkuBF6R5CvAdcB/ADYAlwFL+90cmGT5yG43V9VP9+a4kiRp53bVggc4GrhpO/MAf0R3+/yv6G7XfxN4Wh/Q2wL+sin7uwJYsZfHlSRJO7HLgK+qJ47MP2TKug3Af+ynqe+7AchO9vuyPT2uJEnaOb+LXpKkBhnwkiQ1yICXJKlBBrwkSQ0y4CVJapABL0lSgwx4SZIaZMBLktQgA16SpAYZ8JIkNciAlySpQQa8JEkNMuAlSWqQAS9JUoMMeEmSGmTAS5LUIANekqQGGfCSJDXIgJckqUEGvCRJDTLgJUlqUKpq6BpmTZLbgRtn6XAPAO6YpWPNtlbPzfOaPK2eW6vnBe2e22ye19FVdciuNtqnAn42JVlVVSuGrmMcWj03z2vytHpurZ4XtHtuc/G8vEUvSVKDDHhJkhpkwI/PyqELGKNWz83zmjytnlur5wXtntucOy/74CVJapAteO0zkjT/+57kl5N8LMmRQ9ciaVjNX/C0e5LMS/KsJPv1ry9Ncv7Qdc2QM5N8M8mhowuTbEryrKGKmmHnACcAdw9dyN5Isn+ShdPcduG239dJlOSLSd43dB3jkGS/JPsPXce+yoDvJVmZ5Hn9/BeSnDiy7jFJvp5kXZLLk/xSv/y0JJVk6Q72mSRvTvLDJD9O8qkkB0/Z5mfHSvK9qeEzgEXAh4Cz+9eHAzcMVs3MeiHw/aq6bcrye4G1A9Qzo5I8FXgc8OKqujvJEUkydF176CrgriRr+mlDknv7+fX9tCbJGuAu4AsD17s37gU2DF3Enuqvc89M8uQkT0ry1CQP71e/ALglyaOSPKVf/+QkT0ty/wFr3tn1/uwk1ye5O8nnkxzdLz+mv95Pnc7t1398ZNn6JFcmOX26xx0HA/4+jwCu7uePB74NkOQQ4BLg8n6bC4CLkhw4jX2+Evht4OnAScCDgfdM2eZ44NtJlgAHbid8ZlVVraer8UX9ooOBHwxX0cxI8mjgVOC/bmf1VmDz7FY0s/rfxw8Df1JV3+m7Iz4HXDiJIV9VP1dVS6tqeVUtB/4ceHM//37gvG3r+u1OGbbivbJl6oIkC5IsHqKYPRDg14E30A00eyawLeA3ATcDjwZ+g+5a+qx+Gizg2fH1/mnA24GXAo+i+9l8csp7Hw8cNDKdN7Lugn7ZQ4G/Bj6T5NhdHXdsqmqfnoCPAOvpLvLr+6n6f48G/jPwfe4bkDgPuA14PnBav+3SHez7M8AHR14/Ffjbfv4LU471sxoG+u+QkXNcCuzfL9sMPGvon9MMnN/53a/7dtetAZ4wdI17eX4fobtYLOpfn9n/7J48dG17cU6fBtb1P58NdC3dNSN/L2v69RcPXesMnGdtZ7pq6Np28zxeBny6nz+R7i7gC4Fv9MuW7uhvcBZr3NX1/r3ARSPbPwb4X3R3No/pt33kDvb9ceB/TFl2HfCKXR13XOdrCx7eAjyW7isGTwCeQ3d78ARgNV2r76vV/7SqaivdL8Gd09j3NcBzkpzcv/eiqnpuv+7MKcf6XeCifn4IJwNbk6yj+8S9mu4COh/45Mit0rX97adlA9W525KcQNd62Pb6AUkOTrI8yXK6DzJLt71OcliSBYMVvJuSnAW8GHhJVW1M8gDgT4E/rKpLhq1ur9wLvLZ20oIHfo/uIjnp3s+/bBU+EPjVQSvaOw8GnkL3t7Vu4FpG7ep6fw1wapJnAFTVlVX1K1W1cQ+PtxFYOI3jjsU+H/BVdQvdL+CNVXUN3S2la6vqmqraBBwJ3DLlPW+b5oXzrXS3pL6a5GtJnjKyj5tGj9Uv/ueR+dn2NWB+Ve1f990WfSndB5nlI8vOoLvLMBF91v2t6g/Qnd82X6I7r7v66X7AxSOvV9PdYpvzkvwO3fm9oqqu7Bd/iO7i8bbBCpsZ070+TVwXxHZsqKo1I9NtVXXr0EXthS39NI/ug9qcMI3r/Urgz4C/S/KtJC/Yzm4uG2nwrEnysKkbpBusfAbwEODSaRx3LPb5gO8dxX3/E5rReehuzfyrPrLpqKp7quolwHHAd+n67s+d5nFnVXW2Tln8QuAzU5YfBtw0e5XttccBxwKvHll2IrCwqlJVoRtgd8rI6yV0P685rb+L8lvAG6rqE/2yM+nuxrxoOz/PSbMZOK8fSPd84E39/JnAa/r5dzLh4ydakOQ64F3A6UlW07Xg5wHLgXVJzgG+0W97dZIrd7iz8dvhdbeqtlTVa+muGZcAn0jy8SnvP4Ou5b1tumFk3TP738v1dGNizq6qbX3us3+9H7I/ZC5MwAeBe+j6+NbQfdrc1td3FPB/gPdPec83gNez6z74FcCykdcvo+uDeRDw2SnH2tDXsWbo/yZ9rQ+j+5T5fLouiXn98tczYX2ewKF0d2JqB+sntg8eWDAyfxLd43GPHznvS4FTh65zd8+JfizByLIH9X+Lp2xn+3l0d2EWDl37Hp7vp4F3Dl3HXp7D4XQfov8nXSPg5XRdjm+l638+nG6gXdGF/8MHqnNX1/tfAA4Z2X7bNf4JTK8P/qJ+u3+zO8cd1/nagu/6Rj5KN3LyBLof0O9yX9/IKroLJwBJ5tOF33Rasf8APGPk9UV0txMPomuFjB7rKuC5DNcH/zP9Oa6k+xByMd0I2Vf1qw8DfjhQaXukBn4yYZyqajNAkuPoRvC+qqou69fdRtfn94ns4FHOOeo5wNp+vMeaJGvp/t6Oo3sqYE2Sjf2jc9sek7uNkb9Tza6qWg38FNhUVbdU1ceq6ql0gXl9v/66ftvra7iuyF1d7/879z0iDPBlulvrBzM966rqhqr60W4edyz2+YCvqjuAA4FvVtUNdOF7Rf9D2kzXv/nIJOf0z0NueyTi4pHdHNU/I7ltOqxffiHw+nTP0W977/V0fe0/Gj0WcAjw9X5+MOm+XOSjdBfTV1bX1/4GuhYSdAF/80DlaTuSrKBrqb+jqqY+0vNqutHL75z1wvZQVX2qqvarqmV0fyPn053f/asbB3IscDtwenVjQ5ZV1ZKq+sqAZWuKvqFwGnDZwKX8zDSu9xcCr0jy75IcAfzGuXipAAACdElEQVQRXat79BwOHBmQu3w6H56ncdyxmJiRwmN2NPe1yEfnqaqr+8Fx7wb+E/BPwJOqas3I48VTn2X8Ol3f76vpLqwXA4vpnqV/et03qOJo4KZ+INiyqrprpk9sd6T7kp3z6Z7/fHJ1A0Ooqr8c2ewY4IuzX522J8mzgU/RXYR+LclvAsvoPpAdSPd7tx44O8nf1uSNqj+N7m9pf+CjSS6ne2LgA1X15SEL21194D2U7ucxOq5nCd1THFO/Xjh0P7+xhsAMW57kZXQDkw+hGx8xZwK+t8PrPV2gLwb+iu7v6JvA06rqjpEgn3o+V9B1x+7NccfC/9mMAEjyeuCNwK3As6vqOyPrTqR7xOMo4A/o+ngvH6TQPZRk26CWeTXllz7J3XQfvCauBdif19voBgVue7zx9n76MV0/3zy6cSOfrapzBip1r6T7Bsg/Bf493e34W+n6ez8L/OPUn+lc1D++uJruZzKdercN+HxwVc3ZbrEkxwOvo/swdiRdA+e9dHcs/wJ4B93jYvsBP6mq9F/is7XGOIJctuB1n2/Rtd5fV1VTv8f8brq+oxuAcyct3HvbvhVsCf/6sZ39+mniVPe45Ut2sdmWJI+vqon5OtT+6zyPpbsL8XDgFLrxME8A/pHuA+cZwN8Dm5O8q6rePVC509Lfpl00dB1jcCddl945dF9085Mkn+3XvZvuDuaJdCG/th83sQT4fbpH0jQmtuAlzTlJnkk3EPVa4ErgktrOc+FJFtEF/aqq+t7sVqkdSfLzdE8XXZFkUe35F8VoLxjwkiQ1aJ8fRS9JUosMeEmSGmTAS5LUIANekqQGGfCSJDXo/wMHchEEIjfKugAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_attention(text, aa, head_num=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "txt = \"而佛给客服#|#丰富收到fefdhasjhsdf\\n12312#|#23123\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos = [i.span() for i in re.finditer('#\\|#', txt)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5, 8)\n",
      "(30, 33)\n"
     ]
    }
   ],
   "source": [
    "for i in re.finditer('#\\|#', txt):\n",
    "    print(i.span())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "'tuple' object cannot be interpreted as an integer",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-71-f4ee98422afe>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m8\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: 'tuple' object cannot be interpreted as an integer"
     ]
    }
   ],
   "source": [
    "for i in range((5, 8)):\n",
    "    print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0, 2)\n",
      "(3, 5)\n",
      "(6, 8)\n"
     ]
    }
   ],
   "source": [
    "a = \"123123123\"\n",
    "for i in re.finditer('12', a):\n",
    "    print(i.span())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<_sre.SRE_Match object; span=(0, 2), match='12'>\n",
      "<_sre.SRE_Match object; span=(3, 5), match='12'>\n",
      "<_sre.SRE_Match object; span=(6, 8), match='12'>\n"
     ]
    }
   ],
   "source": [
    "for i in re.finditer('12', a):\n",
    "    print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
